{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualizing Benchmarks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation\n",
    "We start from loading the necessary modules and the data, which is immediately converted to dataframes.\n",
    "\n",
    "Then, we process the data if necessary (usually by converting it into long format, which is compatible with Altair)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import yaml\n",
    "import altair as alt\n",
    "from altair_saver import save\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "chart_dir = 'charts'\n",
    "with open(\"results.yaml\", 'r') as stream:\n",
    "    data = yaml.safe_load(stream)\n",
    "if not os.path.exists(chart_dir):\n",
    "    os.mkdir(chart_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "external, internal = data['external'], data['internal']\n",
    "ext_dset_iter = pd.DataFrame(external['dataset_iteration'])\n",
    "ext_seq_acc = pd.DataFrame(external['sequential_access'])\n",
    "int_compres = pd.DataFrame(internal['compression'])\n",
    "int_rand_acc = pd.DataFrame(internal['random_access'])\n",
    "int_dset_iter = pd.DataFrame(internal['dataset_iteration'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "stacked_ext_dset_iter = ext_dset_iter.set_index('loader').stack().reset_index().rename(columns={'level_1': 'Dataset', 0: 'time'})\n",
    "melt_int_rand_acc = int_rand_acc.melt('batch_size', var_name='Dataset', value_name='Time')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Drawing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_ext_dset_iter = alt.Chart(stacked_ext_dset_iter).mark_bar(\n",
    "    cornerRadiusBottomRight=2,\n",
    "    cornerRadiusTopRight=2\n",
    ").encode(\n",
    "    x=alt.X('time:Q', title=\"Time in seconds (log scale)\", scale=alt.Scale(type='log'), axis=alt.Axis(grid=False)),\n",
    "    y=alt.Y('Dataset:O', axis=None),\n",
    "    color='Dataset:N',\n",
    "    row=alt.Row('loader:N', header=alt.Header(title=None, labelOrient='top'))\n",
    ").properties(\n",
    "    height=alt.Step(25),\n",
    "    width=1000,\n",
    "    title=\"Dataset Iteration (external)\"\n",
    ").configure_view(\n",
    "    stroke='transparent'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "wr = alt.Chart(ext_seq_acc).mark_rect()\n",
    "wr_x = alt.X('dataset:O', title=None)\n",
    "wr_y = alt.Y('framework:O', title=None)\n",
    "wr_props = {'height': alt.Step(25), 'width': 100}\n",
    "read = wr.encode(\n",
    "    y=wr_y,\n",
    "    x=wr_x,\n",
    "    color='read:Q'\n",
    ")\n",
    "write = wr.encode(\n",
    "    y=wr_y,\n",
    "    x=wr_x,\n",
    "    color='write:Q'\n",
    ")\n",
    "graph_ext_seq_acc = (read.properties(**wr_props, title='Read') | write.properties(**wr_props, title='Write')).properties(title=\"Sequential Access\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_int_rand_acc = alt.Chart(melt_int_rand_acc).mark_line().encode(\n",
    "    x=\"batch_size\",\n",
    "    y=\"Time:Q\",\n",
    "    color='Dataset:N'\n",
    ").properties(title=\"Random Access\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_int_dset_iter = alt.Chart(int_dset_iter).mark_circle().encode(\n",
    "    y=alt.Y(\"type\", title=None),\n",
    "    x=alt.X(\"time:Q\", axis=alt.Axis(grid=False), title=\"Time in seconds\"),\n",
    "    color='prefetch_factor:N',\n",
    "    size=alt.Size('batch_size:Q', scale=alt.Scale(type='log'))\n",
    ").properties(title=\"Dataset Iteration (internal)\", height=170).configure_view(strokeOpacity=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-0f5773e6db944e3db9c32d74290ba939\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-0f5773e6db944e3db9c32d74290ba939\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-0f5773e6db944e3db9c32d74290ba939\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300, \"stroke\": \"transparent\"}}, \"data\": {\"name\": \"data-b078ecbe0a213db4e022836883152aff\"}, \"mark\": {\"type\": \"bar\", \"cornerRadiusBottomRight\": 2, \"cornerRadiusTopRight\": 2}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Dataset\"}, \"row\": {\"type\": \"nominal\", \"field\": \"loader\", \"header\": {\"labelOrient\": \"top\", \"title\": null}}, \"x\": {\"type\": \"quantitative\", \"axis\": {\"grid\": false}, \"field\": \"time\", \"scale\": {\"type\": \"log\"}, \"title\": \"Time in seconds (log scale)\"}, \"y\": {\"type\": \"ordinal\", \"axis\": null, \"field\": \"Dataset\"}}, \"height\": {\"step\": 25}, \"title\": \"Dataset Iteration (external)\", \"width\": 1000, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-b078ecbe0a213db4e022836883152aff\": [{\"loader\": \"Hub (remote - Wasabi) .to_pytorch()\", \"Dataset\": \"MNIST\", \"time\": 12.4601}, {\"loader\": \"Hub (remote - Wasabi) .to_pytorch()\", \"Dataset\": \"Places365\", \"time\": 6033.2499}, {\"loader\": \"Hub (remote - S3) .to_pytorch()\", \"Dataset\": \"MNIST\", \"time\": 8.4371}, {\"loader\": \"Hub (remote - S3) .to_pytorch()\", \"Dataset\": \"Places365\", \"time\": 4590.9812}, {\"loader\": \"Hub (local) .to_pytorch()\", \"Dataset\": \"MNIST\", \"time\": 353.3983}, {\"loader\": \"Hub (local) .to_pytorch()\", \"Dataset\": \"Places365\", \"time\": 19751.0882}, {\"loader\": \"PyTorch (local, native)\", \"Dataset\": \"MNIST\", \"time\": 13.9312}, {\"loader\": \"PyTorch (local, native)\", \"Dataset\": \"Places365\", \"time\": 4305.0664}, {\"loader\": \"Hub (remote - Wasabi) .to_tensorflow()\", \"Dataset\": \"MNIST\", \"time\": 10.8668}, {\"loader\": \"Hub (remote - Wasabi) .to_tensorflow()\", \"Dataset\": \"Places365\", \"time\": 5725.523}, {\"loader\": \"Hub (remote - S3) .to_tensorflow()\", \"Dataset\": \"MNIST\", \"time\": 11.8887}, {\"loader\": \"Hub (remote - S3) .to_tensorflow()\", \"Dataset\": \"Places365\", \"time\": 4524.5225}, {\"loader\": \"Hub (local) .to_tensorflow()\", \"Dataset\": \"MNIST\", \"time\": 11.0737}, {\"loader\": \"Hub (local) .to_tensorflow()\", \"Dataset\": \"Places365\", \"time\": 2141.25}, {\"loader\": \"Tensorflow (local, native - TFDS)\", \"Dataset\": \"MNIST\", \"time\": 10.9133}, {\"loader\": \"Tensorflow (local, native - TFDS)\", \"Dataset\": \"Places365\", \"time\": 1051.0044}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.Chart(...)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph_ext_dset_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-baf79b3b1c324fc1bc56fcd48f85dcb2\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-baf79b3b1c324fc1bc56fcd48f85dcb2\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-baf79b3b1c324fc1bc56fcd48f85dcb2\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"hconcat\": [{\"mark\": \"rect\", \"encoding\": {\"color\": {\"type\": \"quantitative\", \"field\": \"read\"}, \"x\": {\"type\": \"ordinal\", \"field\": \"dataset\", \"title\": null}, \"y\": {\"type\": \"ordinal\", \"field\": \"framework\", \"title\": null}}, \"height\": {\"step\": 25}, \"title\": \"Read\", \"width\": 100}, {\"mark\": \"rect\", \"encoding\": {\"color\": {\"type\": \"quantitative\", \"field\": \"write\"}, \"x\": {\"type\": \"ordinal\", \"field\": \"dataset\", \"title\": null}, \"y\": {\"type\": \"ordinal\", \"field\": \"framework\", \"title\": null}}, \"height\": {\"step\": 25}, \"title\": \"Write\", \"width\": 100}], \"data\": {\"name\": \"data-3a77d5cb521fafeae31aef5baa7b4154\"}, \"title\": \"Sequential Access\", \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-3a77d5cb521fafeae31aef5baa7b4154\": [{\"framework\": \"TileDB (local)\", \"read\": 1.3107, \"write\": null, \"dataset\": \"MNIST (70000)\"}, {\"framework\": \"zarr (local)\", \"read\": 0.355, \"write\": null, \"dataset\": \"MNIST (70000)\"}, {\"framework\": \"Hub (remote - Wasabi)\", \"read\": 1.1537, \"write\": null, \"dataset\": \"MNIST (70000)\"}, {\"framework\": \"Hub (local)\", \"read\": 0.0483, \"write\": null, \"dataset\": \"MNIST (70000)\"}, {\"framework\": \"TileDB (local)\", \"read\": 12.6473, \"write\": 35.3081, \"dataset\": \"MNIST (7000)\"}, {\"framework\": \"zarr (local)\", \"read\": 0.3461, \"write\": 1.1027, \"dataset\": \"MNIST (7000)\"}, {\"framework\": \"Hub (remote - Wasabi)\", \"read\": 1.0862, \"write\": 0.7641, \"dataset\": \"MNIST (7000)\"}, {\"framework\": \"Hub (local)\", \"read\": 0.1244, \"write\": 0.6852, \"dataset\": \"MNIST (7000)\"}, {\"framework\": \"TileDB (local)\", \"read\": null, \"write\": null, \"dataset\": \"Places365\"}, {\"framework\": \"zarr (local)\", \"read\": null, \"write\": null, \"dataset\": \"Places365\"}, {\"framework\": \"Hub (remote - Wasabi)\", \"read\": null, \"write\": null, \"dataset\": \"Places365\"}, {\"framework\": \"Hub (local)\", \"read\": null, \"write\": null, \"dataset\": \"Places365\"}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.HConcatChart(...)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph_ext_seq_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-82bb1a44135a40be831fe007daef3669\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-82bb1a44135a40be831fe007daef3669\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-82bb1a44135a40be831fe007daef3669\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"data\": {\"name\": \"data-448f72f6cf6236d115a06fafa47a8ae4\"}, \"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Dataset\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"batch_size\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"Time\"}}, \"title\": \"Random Access\", \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-448f72f6cf6236d115a06fafa47a8ae4\": [{\"batch_size\": 1, \"Dataset\": \"MNIST\", \"Time\": 0.5066}, {\"batch_size\": 2, \"Dataset\": \"MNIST\", \"Time\": 0.4056}, {\"batch_size\": 4, \"Dataset\": \"MNIST\", \"Time\": 0.4138}, {\"batch_size\": 8, \"Dataset\": \"MNIST\", \"Time\": 0.4096}, {\"batch_size\": 16, \"Dataset\": \"MNIST\", \"Time\": 0.4106}, {\"batch_size\": 32, \"Dataset\": \"MNIST\", \"Time\": 0.4046}, {\"batch_size\": 64, \"Dataset\": \"MNIST\", \"Time\": 0.4002}, {\"batch_size\": 128, \"Dataset\": \"MNIST\", \"Time\": 0.4083}, {\"batch_size\": 256, \"Dataset\": \"MNIST\", \"Time\": 0.4075}, {\"batch_size\": 512, \"Dataset\": \"MNIST\", \"Time\": 0.4023}, {\"batch_size\": 1, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.1837}, {\"batch_size\": 2, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.1458}, {\"batch_size\": 4, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.1509}, {\"batch_size\": 8, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.1391}, {\"batch_size\": 16, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.1613}, {\"batch_size\": 32, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.1435}, {\"batch_size\": 64, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.1665}, {\"batch_size\": 128, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.234}, {\"batch_size\": 256, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.2858}, {\"batch_size\": 512, \"Dataset\": \"Omniglot (test)\", \"Time\": 0.2476}, {\"batch_size\": 1, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.8322}, {\"batch_size\": 2, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.9117}, {\"batch_size\": 4, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.7624}, {\"batch_size\": 8, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.7664}, {\"batch_size\": 16, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.7576}, {\"batch_size\": 32, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.7389}, {\"batch_size\": 64, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.7494}, {\"batch_size\": 128, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.7731}, {\"batch_size\": 256, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.7553}, {\"batch_size\": 512, \"Dataset\": \"CIFAR10 (train)\", \"Time\": 0.7511}, {\"batch_size\": 1, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.89}, {\"batch_size\": 2, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.748}, {\"batch_size\": 4, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.7582}, {\"batch_size\": 8, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.756}, {\"batch_size\": 16, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.7358}, {\"batch_size\": 32, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.7644}, {\"batch_size\": 64, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.739}, {\"batch_size\": 128, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.7509}, {\"batch_size\": 256, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.7473}, {\"batch_size\": 512, \"Dataset\": \"CIFAR100 (train)\", \"Time\": 0.7656}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.Chart(...)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph_int_rand_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-3d201e83239845a2af3ce99062f8d31c\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-3d201e83239845a2af3ce99062f8d31c\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-3d201e83239845a2af3ce99062f8d31c\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300, \"strokeOpacity\": 0}}, \"data\": {\"name\": \"data-9288e2d804aa1189d658699f649b1104\"}, \"mark\": \"circle\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"prefetch_factor\"}, \"size\": {\"type\": \"quantitative\", \"field\": \"batch_size\", \"scale\": {\"type\": \"log\"}}, \"x\": {\"type\": \"quantitative\", \"axis\": {\"grid\": false}, \"field\": \"time\", \"title\": \"Time in seconds\"}, \"y\": {\"type\": \"nominal\", \"field\": \"type\", \"title\": null}}, \"height\": 170, \"title\": \"Dataset Iteration (internal)\", \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-9288e2d804aa1189d658699f649b1104\": [{\"batch_size\": 1, \"type\": \"PyTorch\", \"prefetch_factor\": 1, \"time\": 114.8104}, {\"batch_size\": 1, \"type\": \"PyTorch\", \"prefetch_factor\": 4, \"time\": 93.0956}, {\"batch_size\": 1, \"type\": \"PyTorch\", \"prefetch_factor\": 16, \"time\": 96.3225}, {\"batch_size\": 1, \"type\": \"PyTorch\", \"prefetch_factor\": 128, \"time\": 100.2829}, {\"batch_size\": 1, \"type\": \"Tensorflow\", \"prefetch_factor\": 1, \"time\": 26.6553}, {\"batch_size\": 1, \"type\": \"Tensorflow\", \"prefetch_factor\": 4, \"time\": 20.9806}, {\"batch_size\": 1, \"type\": \"Tensorflow\", \"prefetch_factor\": 16, \"time\": 20.6421}, {\"batch_size\": 1, \"type\": \"Tensorflow\", \"prefetch_factor\": 128, \"time\": 23.1414}, {\"batch_size\": 16, \"type\": \"PyTorch\", \"prefetch_factor\": 1, \"time\": 14.0271}, {\"batch_size\": 16, \"type\": \"PyTorch\", \"prefetch_factor\": 4, \"time\": 12.8922}, {\"batch_size\": 16, \"type\": \"PyTorch\", \"prefetch_factor\": 16, \"time\": 12.5523}, {\"batch_size\": 16, \"type\": \"PyTorch\", \"prefetch_factor\": 128, \"time\": 12.5023}, {\"batch_size\": 16, \"type\": \"Tensorflow\", \"prefetch_factor\": 1, \"time\": 11.4632}, {\"batch_size\": 16, \"type\": \"Tensorflow\", \"prefetch_factor\": 4, \"time\": 11.2359}, {\"batch_size\": 16, \"type\": \"Tensorflow\", \"prefetch_factor\": 16, \"time\": 10.9313}, {\"batch_size\": 16, \"type\": \"Tensorflow\", \"prefetch_factor\": 128, \"time\": 11.0235}, {\"batch_size\": 128, \"type\": \"PyTorch\", \"prefetch_factor\": 1, \"time\": 8.9637}, {\"batch_size\": 128, \"type\": \"PyTorch\", \"prefetch_factor\": 4, \"time\": 8.981}, {\"batch_size\": 128, \"type\": \"PyTorch\", \"prefetch_factor\": 16, \"time\": 9.0486}, {\"batch_size\": 128, \"type\": \"PyTorch\", \"prefetch_factor\": 128, \"time\": 8.3433}, {\"batch_size\": 128, \"type\": \"Tensorflow\", \"prefetch_factor\": 1, \"time\": 9.7509}, {\"batch_size\": 128, \"type\": \"Tensorflow\", \"prefetch_factor\": 4, \"time\": 9.7083}, {\"batch_size\": 128, \"type\": \"Tensorflow\", \"prefetch_factor\": 16, \"time\": 10.3689}, {\"batch_size\": 128, \"type\": \"Tensorflow\", \"prefetch_factor\": 128, \"time\": 10.8401}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.Chart(...)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph_int_dset_iter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for graph in graph_ext_dset_iter, graph_ext_seq_acc, graph_int_rand_acc, graph_int_dset_iter:\n",
    "    save(graph, f\"charts/{graph.title.replace(' ', '_')}.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
